#encoding:utf-8
import aiohttp
import asyncio
import pymongo


async def lagou_crawl(url):
    header = {
        'Accept-Encoding': 'gzip',
        'Connection': 'keep-alive',
        'Host': 'gate.lagou.com',
        'Referer': 'https://m.lagou.com/search.html',
        'X-L-JANUS-STRATEGY': '{"strategies":[{"key":"unlimited_deliver","value":"A"}]}',
        'X-L-REQ-HEADER': '{"userToken":"09360ac3be57fc36b61c04228af8094545ee0da85fa9fe9844f469b964972826","reqVersion":71000,"lgId":"008796754926520_1560423518612","appVersion":"7.10.1","userType":0,"deviceType":200}',
        'User-Agent': 'okhttp/3.11.0',
    }
    async with aiohttp.ClientSession() as session:
        try:
            async with session.get(url=url,headers=header) as r:
                req=await r.json(encoding='utf-8')
                if len(req['content'])>0:
                    print(req)
                    await inser_msg(table,req)
        except Exception as e:
            print(e)


async def inser_msg(table,item):
    table.insert_one(dict(item))
    return



if __name__ == '__main__':
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    db = myclient['拉勾网']
    table = db['招聘数据']
    url_format = "https://gate.lagou.com/v1/neirong/positions/similarPositions/{0}/"
    start =22200
    end = 22300
    loop = asyncio.get_event_loop()
    while True:
        try:
            tasks = [lagou_crawl(url_format.format(page)) for page in range(start,end)]
            loop.run_until_complete(asyncio.gather(*tasks))
            asyncio.ensure_future()
            with open('record.txt','w',encoding='utf-8')as w:
                w.write(str(end))
            start=end
            end+=100
            tasks.clear()
        except Exception as e:
            print(e)
            loop.close()
            break

